import os
import sys 
import shelve

def isTextFile(fileName):
    if(fileName.find("cleaned") != -1):
        return True
    return False

FILES =  filter(isTextFile,os.listdir("."))
catdb = shelve.open("../catdb.db")

for eachFile in FILES:

    print "\nFile:",eachFile
    
    rankCounts = {}

    rankNotFoundCount = 0 #No of article that has category which has has more than 1000 articles in its membership
    rankUnknown = 0 # This is number of time we cannot find an rank of article whose category has less than 1000 articles in its membership
    rankedTotal = 0 # This is the number of time we were even in a position to rank i.e. rank > -1 
    n = 0
    
    for eachLine in open(eachFile):
        eachLine = eachLine.strip()
        
        if(len(eachLine.split("\t")) > 4):
            
            rank = int(eachLine.split("\t")[3])
            missing_cat = eachLine.split("\t")[2].strip()
            cats = []

            for eachItem in eachLine.split("\t")[1].split(" "):
                if(eachItem != missing_cat):
                    cats.append(eachItem)
            
            rankedItems = eachLine.split("\t")[4].split(" ")
            prunedRankedItems = []
            
            article = eachLine.split("\t")[0]

            for eachCat in rankedItems:
                if(len(cats) >= 1 and eachCat not in cats):
                    prunedRankedItems.append(eachCat)

            #This is done just once first time the article is encountered
            if(rankCounts.has_key(article) == False):
                rankCounts[article] = 0
                
            if(rank == -1):
                rankUnknown += 1
            else:
                rank = prunedRankedItems.index(missing_cat)

                if(rank > 100):
                    #print article,missing_cat,rank
                    rankUnknown += 1
                    continue
#                    
#
#                    try:
#                        print article,missing_cat,len(catdb[missing_cat].split("|"))
#                    except KeyError:
#                        print "\n",missing_cat," is missing" 
#
#                    continue
                
                rankCounts[article] += (rank + 1)
                rankedTotal += 1
                
        else:
            rankNotFoundCount += 1

        n+= 1 
            
    
    totalOfRank = 0 

    for eachArticle in rankCounts.keys():
        totalOfRank += rankCounts[eachArticle]

    print "Total Number of Test instances,Articles:",str(n),str(len(rankCounts.keys()))
    print "Not Ranked %:",(float(rankNotFoundCount)/n)*100
    print "Rank Unknown %:",(float(rankUnknown)/(n-rankNotFoundCount))*100
    print "Avg Rank:",float(totalOfRank)/rankedTotal
    
